import pymongo
import json
import pandas as pd
#后续加入可视化点击，自动生成配置文件
#每个网站的收集配置
#定义规则类及其属性\n",
# 读取excel文件
df = pd.read_excel('/home/scrapy_config/jianbao.xlsx')
db_all=df['db_all'][0]
urls=df['ad']
classname =df['p1']
classtg =df['p2']
cha=df['p3']
trans=df['trans']
class Rule:
    def test():
        print("init Rule")
class Rulemanager:
    def __init__(self,string):
        self.json_str = json.loads(string)
        self.rule = Rule()
    def manager(self):
        for k,v in self.json_str.items():
            setattr(self.rule, k, v)
#定义数据库对象\n",
class DB:
    def __init__(self,ip,port):
        self.myclinet = pymongo.MongoClient('mongodb://'+str(ip)+":"+str(port)+"/")
    def createdb(self,string):
        self.mydb =self.myclinet[string]
    def setcollection(self,coltring):
        self.collection = self.mydb[coltring]
    def insert(self,string):
        #self.collection.insert_one(string)
         self.collection.update_one({'url': string['url']}, {'$set': string}, True)
    def find(self,string):
        self.collection.find(string)
    def delete(self,string):
        self.collection.delete_many(string)
def upload(rulemanager):
    db= DB("192.168.50.101","27017")
    dbclinet =db.createdb(db_all)
    db.setcollection("rule")
    print(rulemanager.json_str)
    db.insert(rulemanager.json_str)
# 主函数\n"
def update(rules):
    #unittest.main(argv=['first-arg-is-ignored'], exit=False)
    #crawl(url, headers)
    #example
    for rule_dict in  rules:
        print(rule_dict)
        rule_json =json.dumps(rule_dict)
        rulemanager= Rulemanager(rule_json)
        upload(rulemanager)



header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
rules =[]
for index,url in enumerate(urls):
    rule={
        'url':url,
        'header':header,
        'search_fule':['name',classtg[index],{'class': classname[index]},True,None],
        'find_rule':'nothing',
        'save_rule':[db_all,None],
        'page_type':'html',
        'character':cha[index],
        'trans':trans[index]
    }
    #print(rule)
    rules.append(rule)
update(rules)